package com.sena.crawler.run;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.sena.crawler.util.JedisUtilNew;
import com.sena.crawler.util.JedisUtils;
import com.sena.crawler.util.ReadProperties;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.ShardedJedis;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.text.SimpleDateFormat;
import java.util.*;

public class RunInsertSenaSpiderConfigUrl {
    public static void main(String[] args) {
    	insertUrlData() ;
    }

    public static void insertUrlData() {
        JedisUtilNew jedisUtilNew = new JedisUtilNew(ReadProperties.getRedisUrlValue("url"));
        ShardedJedis client = jedisUtilNew.getInstance();
        Set<String> sets = client.smembers("COMMON_LINK");
        System.out.println("=================" + sets.size());

        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式
        String now = df.format(new Date());
        String url = ReadProperties.getJDBCUrlValue("url");
        String sql_user = ReadProperties.getJDBCUrlValue("username");
        String sql_pwd = ReadProperties.getJDBCUrlValue("password");

        int count = 0;
        List<String> addList = new ArrayList<String>();
        for (String obj : sets) {
            System.out.println(obj);
            JSONObject urlVo = JSON.parseObject(obj);

            String tempurl = "";
            String name = "";
            if (urlVo.get("url") != null && urlVo.get("link").toString().indexOf("<") > 0) {  //都有尖括号的过滤
                int pos = urlVo.get("link").toString().indexOf("<");
                tempurl = urlVo.get("link").toString().substring(0, pos);
                System.out.println("tempurl====" + tempurl);
            } else
                tempurl = urlVo.get("link").toString();
            if (null != urlVo.get("linkName")) {
                name = urlVo.get("linkName").toString().replaceAll("'", "");
            }
            String insertSql = "insert into sena_spider_config_url(config_id,url,pLink,name,stateus,update_time) values(1,'" + tempurl.replaceAll("'", "") + "','" + urlVo.get("pLink") + "','" + name + "',0,'" + now + "');";
            //  System.out.println(insertSql);
            Connection conn = null;
            Statement stmt = null;


            addList.add(insertSql);
            //  System.out.println(insertSql);
            count++;
            if (addList.size() >= 2000 || count == sets.size()) { //每1000条插入一次
                try {
                    Class.forName("com.mysql.jdbc.Driver");
                    conn = DriverManager.getConnection(url, sql_user, sql_pwd);
                    stmt = conn.createStatement();
                    /* 执行SQL语句，如果conn没有设置自动提交属性，此处需要进行提交 */

                    conn.setAutoCommit(false);
                    stmt = conn.createStatement();

                    for (int i = 0; i < addList.size(); i++) {
                        stmt.addBatch(addList.get(i).toString());
                    }
                    stmt.executeBatch();
                    conn.commit();

                    System.out.println ("================成功插入"+count+"条");
                }
                catch (Exception e){
                    e.printStackTrace();
                }
                finally{
                    addList.clear();
                    if (conn != null){
                        try {
                            conn.close ();
                            //System.out.println ("Database connection terminated");
                        } catch (Exception e) { /* ignore close errors */}
                    }
                    if (stmt != null){
                        try {
                            stmt.close ();
                            //System.out.println ("Database connection terminated");
                        } catch (Exception e) { /* ignore close errors */}
                    }
                }

            }
        }


    }

}
